package com.gxljc.bear.crawler.itaogao.jjrb;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.itaogao.NewsBaseCrawler;
import com.gxljc.bear.crawler.itaogao.NewsBaseTable;
import com.gxljc.bear.crawler.util.DateUtil;

/**
 * 
 * @author Administrator -> haitao
 * @since 2021年9月20日 下午5:08:54
 */
public class JjrbDetailCrawler extends NewsBaseCrawler {

	private static final long serialVersionUID = 8341750078282940682L;
	private static final Logger LOG = Logger.getLogger(JjrbDetailCrawler.class);
	private static final String EDITION_URL_LIST = "http://paper.ce.cn/jjrb/html/%s/node_2.htm";
	private static final String PAGE_URL_LIST = "http://paper.ce.cn/jjrb/html/%s/%s";
	private static final String DEFAULT_DATE_FORMAT = "yyyy-MM/dd";

	public JjrbDetailCrawler() {
		super("t_newspaper_jjrb");
	}

	@SuppressWarnings("unchecked")
	@Override
	protected List<String> genSeeds() {
		LOG.info("--- genSeed of Jjrb ----");
		List<String> seeds = new ArrayList<>();
		try {

			for (String date : genDatesTillNow(DEFAULT_DATE_FORMAT)) {
				// fetch url list by parser
				String url = String.format(EDITION_URL_LIST, date);
				String editionResult = "";
				Pair<String, byte[]> pair = fetResult(url, proxy);
				if (pair == null) {
					continue;
				}
				editionResult = pair.first;
				if (editionResult != null) {
					JSONObject jsonResult = (JSONObject) JSONObject.parse(editionResult);
					if (jsonResult != null) {
						List<String> urlList = (List<String>) jsonResult.get("edition_url");
						LOG.info("urlList" + urlList);
						if (urlList != null) {
							for (String s : urlList) {
								String nodeUrl = String.format(PAGE_URL_LIST, date, removeSuffix(s));
								String nodeResult = "";
								Pair<String, byte[]> pair2 = fetResult(nodeUrl, proxy);
								if (pair2 == null) {
									continue;
								}
								nodeResult = pair2.first;
								if (nodeResult != null) {
									JSONObject json2 = (JSONObject) JSONObject.parse(nodeResult);
									if (json2 != null) {
										List<String> pageUrl = (List<String>) json2.get("page_url");
										if (pageUrl != null) {
											seeds.addAll(completeUrl(PAGE_URL_LIST, date, pageUrl));

										}
									}
								}
							}
						}
					}
				}

			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		LOG.info("total size=" + seeds.size());
		return seeds;
	}

	@SuppressWarnings("unchecked")
	@Override
	protected List<String> genNewSeeds() {
		String date = DateUtil.dateFormat(new Date(), "yyyy-MM/dd");
		String request = String.format(EDITION_URL_LIST, date);
		LOG.info("--- genNewSeed of Jjrb ----");
		List<String> seeds = new ArrayList<>();
		try {

			String editionResult = "";
			Pair<String, byte[]> pair = fetResult(request, proxy);
			if (pair == null) {
				return seeds;
			}
			editionResult = pair.first;
			if (editionResult != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(editionResult);
				if (jsonResult != null) {
					List<String> nodeList = (List<String>) jsonResult.get("edition_url");
					LOG.info("nodeList" + nodeList);
					if (nodeList != null) {
						for (String s : nodeList) {
							String nodeUrl = String.format(PAGE_URL_LIST, date, removeSuffix(s));
							String nodeResult = "";
							Pair<String, byte[]> pair2 = fetResult(nodeUrl, proxy);
							if (pair2 == null) {
								continue;
							}
							nodeResult = pair2.first;
							if (nodeResult != null) {
								JSONObject json2 = (JSONObject) JSONObject.parse(nodeResult);
								if (json2 != null) {
									List<String> pageUrl = (List<String>) json2.get("page_url");
									if (pageUrl != null) {
										seeds.addAll(completeUrl(PAGE_URL_LIST, date, pageUrl));

									}
								}
							}
						}
					}
				}
			}

		} catch (Exception e) {
			e.printStackTrace();
		}
		LOG.info("total size=" + seeds.size());
		return seeds;

	}

	@SuppressWarnings("unchecked")
	@Override
	protected Pair<NewsBaseTable, byte[]> genTable(String url) {

		NewsBaseTable table = new NewsBaseTable();
		String request = url;
		String result = "";
		Pair<String, byte[]> pair = fetResult(request, proxy);
		if (pair == null) {
			return new Pair<NewsBaseTable, byte[]>(table, "".getBytes());
		}
		result = pair.first;
		byte[] html = pair.second;
		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			if (jsonResult != null) {

				String title = (String) jsonResult.get("title");
				if (!StringUtil.isNullOrEmpty(title)) {
					table.setTitle(title);
				}
				table.setPublishDate(extractDate(url));
				List<String> images = (List<String>) jsonResult.get("pic_list");
				LOG.info("images=" + images);
				if (validateObject(images)) {
					table = processImages(url, table, images);
				}
				List<String> paraList = (List<String>) jsonResult.get("para_list");
				if (paraList != null) {
					table.setContent(genContent(paraList, table.getImages()));
				}
				String subTitle = (String) jsonResult.get("sub_title");

				if (!StringUtil.isNullOrEmpty(subTitle)) {
					table.setSubTitle(subTitle);
				}

				String author = (String) jsonResult.get("author");
				if (!StringUtil.isNullOrEmpty(author)) {
					table.setAuthor(author);
					table.setSource(author);
				}
				// set default value
				table.setId(StringUtil.genMD5Val(url));
				table.setCrawlUrl(url);
				table.setCrawlDate(new Date());
				table.setCrawlFlag(1);
			}
		}
		return new Pair<NewsBaseTable, byte[]>(table, html);
	}

	/**
	 * 
	 * @param args
	 * @throws Exception
	 *             attention： 1)
	 *             "../../../res/1/1/2016-08/16/01/res10_attpic_brief.jpg" ->
	 *             http://paper.ce.cn/jjrb/res/1/1/2016-08/16/01/
	 *             res13_attpic_brief.jpg 2） "./node_3.htm" -> /node_3.htm
	 */
	// public static void main(String[] args) throws Exception {
	// String url =
	// "http://paper.ce.cn/jjrb/html/2016-08/16/content_309133.htm";
	// url = "http://paper.ce.cn/jjrb/html/2016-08/15/content_309050.htm";
	// // url = "http://paper.ce.cn/jjrb/html/2016-08/16/node_3.htm";
	// JjrbDetailCrawler crawler = new JjrbDetailCrawler();
	// Pair<String, byte[]> result = crawler.fetResult(url, false);
	// if (result != null) {
	//
	// System.out.println(result.first);
	// }
	// }

}
